package main

import (
	"fmt"
	goq "lib/goquery"
	"lib/request"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	main_domain = "http://bj.5i5j.com" //主域名
	LOG         = "5i5j"
)

var (
	waitgroup = &sync.WaitGroup{}
	NumCPU    = runtime.NumCPU()
)

func init() {
	runtime.GOMAXPROCS(NumCPU)
}

func main() {
	getDistrict("/broker/")

	s(time.Now().Unix(), "done.")
}

//获取区县
func getDistrict(url string) {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog(fmt.Sprintf("抓取 区县列表[%s] 失败 :%s", tmpurl, e.Error()))
		return
	}

	doc.Find(".keywords > a").NotSelection(doc.Find(".keywords > a").Eq(0)).Each(func(i int, sec *goq.Selection) {
		var (
			district string
			link     string
		)
		link, ok := sec.Attr("href")
		if ok {
			district = filterStr(sec.Text())
			getBlock(link, district)
		}
	})
}

//获取板块列表
func getBlock(url string, district string) {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog(fmt.Sprintf("抓取 热点列表[%s] 失败 :%s", tmpurl, e.Error()))
		return
	}

	doc.Find(".keywords01 a").Each(func(i int, sec *goq.Selection) {
		var (
			block string
			link  string
		)
		link, ok := sec.Attr("href")
		if ok {
			block = filterStr(sec.Text())
			getAgentlist(link, block, district)
		}
	})
}

func getAgentlist(url string, block, district string) {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		s(fmt.Sprintf("抓取 经纪人列表[%s] 失败 :%s", tmpurl, e.Error()))
		return
	}

	//获取页码
	tmppage := filterStr(doc.Find(".zhiye_top").Text())
	tmppage = strings.Replace(tmppage, "我爱我家", "", -1)
	tmppage = strings.Replace(tmppage, "名置业顾问，帮您轻松安家", "", -1)
	recordCount, e := strconv.Atoi(tmppage)

	if e != nil {
		errlog(fmt.Sprintf("抓取 经纪人列表总条数[%s] 失败 [%s]", tmpurl, e.Error()))
		return
	}

	if recordCount == 0 {
		return
	}

	var pageSize = 10
	var pageCount = int(math.Ceil(float64(recordCount) / float64(pageSize)))
	for i := 1; i <= pageCount; i += NumCPU {
		for j := i; j < i+NumCPU && j <= pageCount; j++ {
			waitgroup.Add(1)
			tmpstr := fmt.Sprintf("%sn%d?images", url, j)
			go getAgentList(tmpstr, block, district)
		}
		waitgroup.Wait()
		time.Sleep(time.Second * 3)
	}
}

//获取经纪人列表并找到第一条房源
func getAgentList(url string, block, district string) {
	defer waitgroup.Done()
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog(fmt.Sprintf("抓取 经纪人列表[%s] 失败 [%s]", tmpurl, e.Error()))
		return
	}
	doc.Find(".mt6 .zhiye_content .rightconts").Each(func(i int, sec *goq.Selection) {
		agent_link, ok := sec.Find(".righttuijian .leftimgsty a").Eq(0).Attr("href")
		if ok {
			getAgent(agent_link, block, district)
		}
	})
}

//获取经纪人信息
func getAgent(url string, block, district string) {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog(fmt.Sprintf("抓取 房源详情页[%s] 失败 [%s]", tmpurl, e.Error()))
		return
	}
	var (
		shop_name    string //门店名称
		broker_name  string //经纪人姓名
		mobile_phone string //手机
	)

	shop_name = filterStr(doc.Find(".house-broker-info dt p:contains('所属门店 :') a").Text())
	broker_name = filterStr(doc.Find(".house-broker-info dd .mr-t").Text())
	mobile_phone = filterStr(doc.Find(".house-broker-info dd .house-broker-tel").Text())

	if "" == shop_name || "" == broker_name || "" == mobile_phone {
		return
	}

	str := district + "||" + block + "||" + shop_name + "||" + broker_name + "||" + mobile_phone
	outlog(str)
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := request.Get(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.Replace(str, ",", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//debuglog
func debuglog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_debug.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
